import { PlaywrightCrawler, Request } from 'crawlee'
import { fetchCompanyJobs, updateCompanyJobs } from '../services/jobs.js'
import { gongsiUrl } from '../common/urls.js'
import { writeHtml } from '../common/utils.js'

const setup = async () => {
    const jobs = await fetchCompanyJobs()
    const requestQueue = jobs.map((job: any) => {
        const url = gongsiUrl(job)
        return new Request({ url, userData: { job } })
    })
    const crawler = new PlaywrightCrawler({
        async requestHandler({ page, request, log }) {
            const job = request.userData.job
            const selector = '.company-info-box'
            await page.waitForSelector(selector)
            const html = await page.content()
            writeHtml(job.id, 'companys', html)
            updateCompanyJobs(job)
            log.info(`OK = > ${job.id}`)
        }
    })
    await crawler.run(requestQueue)
}

export default setup